from pyspark.context import SparkContext, SparkConf
from pyspark.sql import HiveContext
import sys
import copy
import pandas as pd

sparkconf = SparkConf().setAppName("liz_pred").set("spark.ui.showConsoleProgress", "false")
sc.setLogLevel('ERROR')
hiveCtx = HiveContext(sc)

gid_phone = hiveCtx.sql("select * from bi_dim.gt_phonemodel_attr_final b")
gid_phone

db = 'liz'
hiveCtx.sql("use {db}".format(db=db))

gid_list = [
    'ANDROID-08e849b0811a469c83f910fed8b86ac0',
    'ANDROID-09168e8b0a42479383f3bd7e4a47a5a3',
    'ANDROID-0c67f11b81759da3c9c6cc9436b77a20',
    'ANDROID-11e3339a4cc04c678fcb812659d9a28c',
    'ANDROID-13b0a6c70e8345c096cd56621d99f0dd',
    'ANDROID-1d708b28422af7bd923520b65d738586',
    'ANDROID-2039059edf0b4d4db96def789e99ddae',
    'ANDROID-231ebf034ad1452aac8b63b4d9b0e137',
    'ANDROID-2846c5aefe3648f2a47d788d2cf054ac',
    'ANDROID-29ee8eb3c98e364b1f4a4068aa57f74a',
    'ANDROID-2f3404cbb4cd48cbb6370edcd726c49a',
    'ANDROID-3cb930b9f911493b819b11f72426c0f6',
    'ANDROID-3ef5f94912da4f308c737d013a2a6e62',
    'ANDROID-47338206c2d2482b994fb454f186caef',
    'ANDROID-48c7549fb12749fe9ac3a8c0a08e8895',
    'ANDROID-534b4fd6115d4ca8a8f63d79fa337508',
    'ANDROID-535b60573e9a4c53b4b7bb5fb1bd2c6a',
    'ANDROID-545457a542f57cf6e530b48f644034f3',
    'ANDROID-57b03e557d084e56a561627d8c870d9d',
    'ANDROID-581e91572f1d42868888c0e67a0abaed',
    'ANDROID-7b98d2af43ee41e6b82b52c8347add91',
    'ANDROID-868b37edcb34466984c8c12dfcfd301d',
    'ANDROID-8857bd9add744264a0654a232b3f8d34',
    'ANDROID-88dc54b88d1f40959b2ed2f34e0466b0',
    'ANDROID-8f0eaf1e185b41a6a735cf9b35238bf3',
    'ANDROID-90ac79914bef4f8eab762ce94243a85c',
    'ANDROID-91baff15df9b403991de2a4a63aaf303',
    'ANDROID-9363f79883074caa8cc82b4d8b718e88',
    'ANDROID-94199791829147eb9f7f013819503d88',
    'ANDROID-9446283586d7482a80d209736bf7ff98',
    'ANDROID-9477b930374f413584730c3dffb8cb43',
    'ANDROID-98b829845a454ebcbeb5ef36e2f69609',
    'ANDROID-9e8f788d9003416e830d70470c8e7764',
    'ANDROID-a714f48a4c8a4cfc8d27fcbd1917bb30',
    'ANDROID-a8ac3e9d176649fbbc4c8148ea95323a',
    'ANDROID-a913aa1dd1a84dfc8355649e7fa3f9b3',
    'ANDROID-aad68284031e42bea2f119feec223374',
    'ANDROID-abe2ddf96fb743c48ece4f0687d84c80',
    'ANDROID-ac5f4d4e04b34a2a9a4e45ca73cb586a',
    'ANDROID-ae708a16cb664919865c4d5300305068',
    'ANDROID-b29237584a1d4a22ae231f462fa21745',
    'ANDROID-b4078e93c0f84aa2aa9f040dc7429ea4',
    'ANDROID-b8fc84219eec4f2aa1aa5500dd690a28',
    'ANDROID-bbfa5ed8fcc448648aced42712b96036',
    'ANDROID-bcd12bc213804a1c8430108f164358fb',
    'ANDROID-bd4590e803ff46fd8d9d2d5b06849629',
    'ANDROID-ca755980d0ca45c8b92342a40526f3c3',
    'ANDROID-ce46baca15f245c8ac9e052530456e86',
    'ANDROID-d5c1705dd548471dabfa0cec96f562a8',
    'ANDROID-d5df710642484bd4b28b9317be0cd472',
    'ANDROID-de4e796eaf2f418c89fc404152d94ba3',
    'ANDROID-de73cb7a98d247db863b8a88c9012977',
    'ANDROID-e69f4aecffff45779b3bb399eda40904',
    'ANDROID-f69e9fcc74374587a772f4e8015c0859',
    'ANDROID-fe04120837bc4be789e96e40ddb69417',
    'ANDROID-ff46b2a2d3f044c4ae0c6b4d181588d3'
]

gid_list = [
    'ANDROID-0fc5ba5fad2510b406e02e9fa3c809c9',
    'ANDROID-14e0d19c75be4be8b124ec117d7072d0',
    'ANDROID-190a7c85d6894ba2b1e27070a669c02b',
    'ANDROID-29c7a3700fefce3d5d89ec9affc180ca',
    'ANDROID-3068329021bf4585b22065e2942b5202',
    'ANDROID-34b2b08a3c1efb7074698adcc4f92cde',
    'ANDROID-355262c88693886bf3a443ee51e0edbe',
    'ANDROID-36c9e01f01984fd9b8c8b31f97588f3d',
    'ANDROID-3a9e9c372f6439da980215493107d606',
    'ANDROID-3d55d4457c074e678aac5bcc964420d6',
    'ANDROID-54c1b31255a349c98e2668b3bf2a0557',
    'ANDROID-66614408c7e24840935d33c9b8c7c523',
    'ANDROID-6c261517678543b8b21dfe4ddd0883c8',
    'ANDROID-863877e661474bbaac47dd02e310e557',
    'ANDROID-98a670c3f89e40b0b7baa59ee68cc8e5',
    'ANDROID-9acb54d0c44c4c398c0cf3e1fb431c84',
    'ANDROID-a2c04d1efd0bc202bdb0178083aac130',
    'ANDROID-ade9c89a7bd5726d17fa1d36d6fddd6f',
    'ANDROID-ae310ca4ecc3438b9d2c8f15488a39d0',
    'ANDROID-b4d25eac0ef740ce85535874a6020a3b',
    'ANDROID-bfca474986ab46c4a8f1a427f52a8bdb',
    'ANDROID-c824da0cfeac33c5c041bfdf7aed1d86',
    'ANDROID-ca344defa644418f9e04efe4dd264226',
    'ANDROID-cd49e118d0ae677fb2db4daa587014de',
    'ANDROID-ce3efe8e00bc411cac8a527b8d57e503',
    'ANDROID-d1b5599fac6a4ce6ac468ecaa11fec22',
    'ANDROID-de7570765163445c9c7ba738195aff15',
    'ANDROID-e2225ad46eae40fd9375845757249d6e',
    'ANDROID-ea48eef4f6204872b250f274f1e2b960',
    'ANDROID-ed47a76d696047b9919b89b3d965ecc1',
    'ANDROID-eef39f4f045f48ea822dff9b32d75ed9',
    'ANDROID-f2365f76286143f0986fd54768451b14',
    'ANDROID-f4ed336cfe4047dc88faf925d0ca3017',
    'ANDROID-f9a69076c53941b990015803e8b67b48',
    'ANDROID-fd07ec51da959ac3466621e39ad16611'
]

GIDpd = pd.DataFrame(gid_list, columns=['gid'])
df_tmp = spark.createDataFrame(GIDpd)
df_tmp.show()

df_tmp_pd = df_tmp.toPandas()
df_tmp_pd['day'] = '20191210'

df_tmp_pd['day'] = df_tmp_pd['day'].apply(lambda x: str(x))
df_tmp_pd['matched_gid'] = df_tmp_pd['gid']
df_tmp_df = spark.createDataFrame(df_tmp_pd)
df_tmp_df.show()

from gai.v2.spark.transformer import DayRectifier

day_rectifier = DayRectifier(inputCol='day', outputCol='rectified_day')
rectified_day_df = day_rectifier.transform(df_tmp_df)
print(rectified_day_df.show())

cols = [
    'ft_usertags'
    , 'ft_installapp_ls'
    , 'ft_brand'
    , 'ft_phone_model'
]

from gai.v2.spark.transformer import FeatureRetriever

feature_retriever = FeatureRetriever(inputIdCol="matched_gid",
                                     inputDayCol="rectified_day",
                                     outputFeatureCols=cols,
                                     extraParams={'span.in.months': 1})
retrieved_feature_df = feature_retriever.transform(rectified_day_df)

res_tb = 'df_tmp_res'
retrieved_feature_df.write.saveAsTable('df_tmp_res', None, "overwrite", None)

gid_phone = hiveCtx.sql("select a.gid,b.phone_model,b.phone_price \
                        from df_tmp_res a left outer join bi_dim.gt_phonemodel_attr_final b \
                        on a.ft_phone_model=b.phone_model")

gid_phone.show(100)
